package cc.spider.demo.web;

import cc.spider.core.SpiderCore;
import cc.spider.web.WebSpiderConfig;
import cc.spider.web.model.GlobalDefaultDataExt;
import cc.spider.web.model.http.Header;
import cc.spider.web.model.http.Request;

public class WebMain {
    public static void main(String[] args) {
        WebSpiderConfig webSpiderConfig = new WebSpiderConfig();
        SpiderCore spiderCore = new SpiderCore(webSpiderConfig);
        GlobalDefaultDataExt globalDefaultData = new GlobalDefaultDataExt();
        globalDefaultData.addHeaders(new Header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36"));
        webSpiderConfig.setGlobalDefaultData(globalDefaultData);
        Request request = new Request();
        request.setUrl("https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd=%E8%AE%B8%E5%B5%A9");
        spiderCore.addTask(request);
        spiderCore.start();
    }
}
